﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace UniversityPhoneCrawler
{
    class Program
    {
        static async Task Main(string[] args)
        {
            Console.WriteLine("高校电话号码爬虫");
            Console.Write("请输入搜索关键字: ");
            string keyword = Console.ReadLine();

            var crawler = new PhoneCrawler();
            await crawler.StartAsync(keyword);
        }
    }

    public class PhoneCrawler
    {
        private readonly Regex _phoneRegex = new Regex(
            @"(?:0\d{2,3}[-—]\d{7,8}|\d{4}[-—]\d{7,8}|$0\d{2,3}$\d{7,8}|400[-—]\d{3}[-—]\d{4})",
            RegexOptions.Compiled);

        private readonly ConcurrentDictionary<string, HashSet<string>> _phoneToUrls =
            new ConcurrentDictionary<string, HashSet<string>>();

        private readonly ConcurrentBag<string> _crawledUrls = new ConcurrentBag<string>();
        private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(5); 
        private readonly HttpClient _httpClient = new HttpClient();
        private CancellationTokenSource _cancellationTokenSource;
        private int _maxPhones = 100;
        private int _currentPhones = 0;

        public PhoneCrawler()
        {
            _httpClient.Timeout = TimeSpan.FromSeconds(8);
            _httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
        }

        public async Task StartAsync(string keyword)
        {
            _cancellationTokenSource = new CancellationTokenSource();
            var searchUrls = await BaiduSearchAsync(keyword);

            Console.WriteLine($"获取到 {searchUrls.Count} 个待爬取URL");
            _currentPhones = 0;

            var tasks = searchUrls.Select(url => ProcessPageAsync(url, _cancellationTokenSource.Token));
            await Task.WhenAll(tasks);

            PrintResults(keyword);
        }

        private async Task<List<string>> BaiduSearchAsync(string keyword)
        {
            var results = new List<string>();
            int pages = 2; 
            string baseUrl = "https://www.baidu.com/s";

            for (int i = 0; i < pages; i++)
            {
                try
                {
                    string url = $"{baseUrl}?wd={Uri.EscapeDataString(keyword)}&pn={i * 10}";
                    string html = await _httpClient.GetStringAsync(url);

                    var matches = Regex.Matches(html, @"data-tools='{""url"":""(.*?)""}");
                    foreach (Match match in matches)
                    {
                        if (match.Success && match.Groups[1].Value.StartsWith("http"))
                        {
                            string decodedUrl = Uri.UnescapeDataString(match.Groups[1].Value);
                            results.Add(decodedUrl);
                        }
                    }

                    await Task.Delay(1000);
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"百度搜索出错: {ex.Message}");
                }
            }

            return results.Distinct().ToList();
        }

        private async Task ProcessPageAsync(string url, CancellationToken cancellationToken)
        {
            if (Volatile.Read(ref _currentPhones) >= _maxPhones || cancellationToken.IsCancellationRequested)
                return;

            try
            {
                await _semaphore.WaitAsync(cancellationToken);

                string html = await _httpClient.GetStringAsync(url);
                _crawledUrls.Add(url);
                Console.WriteLine($"已爬取: {url}");

                ExtractPhoneNumbers(html, url);
            }
            catch (Exception)
            {
            }
            finally
            {
                _semaphore.Release();
            }
        }

        private void ExtractPhoneNumbers(string html, string sourceUrl)
        {
            var matches = _phoneRegex.Matches(html);
            foreach (Match match in matches)
            {
                if (!match.Success || Volatile.Read(ref _currentPhones) >= _maxPhones)
                    continue;

                string phone = match.Value
                    .Replace("—", "-")
                    .Replace("(", "")
                    .Replace(")", "");

                if (_phoneToUrls.TryGetValue(phone, out var urls))
                {
                    lock (urls)
                    {
                        urls.Add(sourceUrl);
                    }
                }
                else if (_phoneToUrls.TryAdd(phone, new HashSet<string> { sourceUrl }))
                {
                    if (Interlocked.Increment(ref _currentPhones) >= _maxPhones)
                    {
                        _cancellationTokenSource.Cancel();
                    }
                }
            }
        }

        private void PrintResults(string keyword)
        {
            Console.WriteLine("\n===== 爬取结果 =====");
            Console.WriteLine($"已爬取URL总数: {_crawledUrls.Count}");
            Console.WriteLine($"有效电话号码数量: {_phoneToUrls.Count}");

            Console.WriteLine("\n电话号码示例及其来源:");
            foreach (var kvp in _phoneToUrls.Take(5))
            {
                Console.WriteLine($"\n电话号码: {kvp.Key}");
                foreach (var url in kvp.Value.Take(2))
                {
                    Console.WriteLine($"  - {url}");
                }
            }

            string fileName = $"{keyword.Replace(" ", "_")}_phone_results.txt";
            using (var writer = new System.IO.StreamWriter(fileName, false, Encoding.UTF8))
            {
                writer.WriteLine($"关键字: {keyword}");
                writer.WriteLine($"爬取时间: {DateTime.Now}");
                writer.WriteLine($"电话号码总数: {_phoneToUrls.Count}\n");

                foreach (var kvp in _phoneToUrls)
                {
                    writer.WriteLine($"电话号码: {kvp.Key}");
                    foreach (var url in kvp.Value)
                    {
                        writer.WriteLine($"  来源URL: {url}");
                    }
                    writer.WriteLine();
                }
            }

            Console.WriteLine($"\n完整结果已保存至: {fileName}");
        }
    }
}